from scipy import sparse
import scipy
#import pickle
import os

class CatWordsMatrix(object):
    """
    Classe contenant 4 choses:
    - matrixNbFiles (nbCats x nbWords): Pour chacunes des categories, on note le nombre de fichiers contenant au moins une fois chacun des mots
    - matrixFreq (nbCats x nbWords): Pour chacunes des categories, on note la frequence totale de chacun des mots
    - categories (nbCats): Pour chacunes des categories, on note son nom et son nombre de fichiers par cle ID
    - words (nbWords): Pour chacuns des mots, on note son nom et sa frequence totale par cle ID
    """
    def __init__(self, DATA_PATH):
        self.DATA_PATH = DATA_PATH
        self.matrixNbFiles = sparse.lil_matrix((1, 1), dtype='int32')
        self.matrixFreq = sparse.lil_matrix((1, 1), dtype='float64')        
        self.categories = {}
        self.words = {}
        self.retraceWords = {}
        
        
    def loadMatrix(self, cats, words, files):
        print 'Build Cats'
        for c in cats.bag.items():
            cId = c[1][0]
            cFreq = c[1][1]
            cName = c[0]
            self.categories[cId] = [cName, cFreq]
        print 'Build Words'
        for w in words.bag.items():
            wId = w[1][0]
            wFreq = w[1][1]
            wName = w[0]
            self.words[wId] = [wName, wFreq]

        self.matrixNbFiles = sparse.lil_matrix((cats.bag.__len__(), words.bag.__len__()), dtype='int32')
        self.matrixFreq = sparse.lil_matrix((cats.bag.__len__(), words.bag.__len__()), dtype='float64')           
        print 'Build Matrix'
        nbFiles = files.bag.__len__()
        for f in files.bag.items():
            cId = cats.bag[f[1][2]][0]
            nbFilesInCat = self.categories[cId][1]
            for w in f[1][1].items():
                wName = w[0]
                if words.bag.has_key(wName) :
                    wId = words.bag[wName][0]
                    wFreq = w[1]
                    curNb = self.matrixNbFiles[cId, wId]
                    curFreq = self.matrixFreq[cId, wId]
                    freqTot = curFreq + wFreq
                    nb = curNb + 1
                    self.matrixNbFiles[cId, wId] = nb
#                    self.matrixFreq[cId, wId] = freqTot                    
                    self.matrixFreq[cId, wId] = freqTot * float(nb) / nbFilesInCat
#freq1             self.matrixFreq[cId, wId] = (float(freqTot)/nbFilesInCat)
#freq2             self.matrixFreq[cId, wId] = freqTot * (float(nbFilesInCat)/nbFiles)
#freq3             self.matrixFreq[cId, wId] = freqTot * (float(nb)/nbFiles)
#freq4             self.matrixFreq[cId, wId] = (float(freqTot)/nbFilesInCat) * (float(nb)/(nbFilesInCat-nb+1))          
        
            
    def pruneMatrix(self, min_var):
        sdevs = scipy.zeros(self.matrixFreq.shape[1])
        for i in range(self.matrixFreq.shape[1]):
            sdevs[i] = self.matrixFreq[:, i].toarray().std()
        
        liminfSdev = (sdevs.mean() + sdevs.std()/2.0)*0.75
        
        notPrune = []
        for i in range(self.matrixFreq.shape[1]):
            if sdevs[i] > liminfSdev:
                self.retraceWords[i] = notPrune.__len__()                
                notPrune.append(i)
                              
        self.matrixFreq = self.matrixFreq[:, notPrune]
        self.matrixNbFiles = self.matrixNbFiles[:, notPrune]        
        print 'End of pruning'
        

